package com.xy.annotaion.handler.kotlin.spider

import cn.hutool.core.io.FileUtil
import com.xy.annotaion.handler.kotlin.p
import org.jsoup.Jsoup

const val MAX_SIZE = 1024 * 1024 * 100
const val PATH = "D:/wintools/"
fun main() {


    val home = "https://live.sysinternals.com"

    var doc = Jsoup.connect(home).maxBodySize(MAX_SIZE).get();
    var aList = doc.body().select("pre").select("a");

    for (aElem in aList) {
        p(aElem)
        var dPath = aElem.attr("href");
        p(dPath)
        var dUrl = home + dPath
        p("download url : $dUrl")
        var filename = aElem.text()
        p("filename : $filename")
        var fileAbsPath = PATH + filename;
        if (FileUtil.exist(fileAbsPath)) {
            continue;
        }

        try {
            FileUtil.writeBytes(
                Jsoup.connect(dUrl)
                    .ignoreContentType(true)
                    .maxBodySize(MAX_SIZE)
                    .execute()
                    .bodyAsBytes(),
                fileAbsPath
            )
        } catch (e: Exception) {
            e.printStackTrace()
        }
        p("DONE: $fileAbsPath")
    }

}
